import puppeteer from 'puppeteer';
import fs from 'fs';


const browser = await puppeteer.launch({
  headless: false,
  defaultViewport: null,
  // args: ['--start-maximized'],
  ignoreDefaultArgs: ['--enable-automation']
});

const page = await browser.newPage();
await page.goto('https://u.gitcafe.net/#hyds');
const trs = await page.$$('.tableizer-table.key_nav tr');
let data = [];


async function scrap() {
  for (let tr of trs) {
    await traverse(tr);
  }
  fs.writeFileSync('data.json', JSON.stringify(data));

}

async function traverse(tr) {
  const head = await tr.$eval('td:first-child', el => el.innerText);
  const table_items = await tr.$$('td:not(:first-child) a');
  for (let item of table_items) {
    item.click();
    await page.waitForTimeout(1000);
    await page.waitForSelector('.GeneratedTable.list tbody tr');
    const tabData = await page.$$eval('.GeneratedTable.list tbody tr', els => els.map(el => {
      return {
        data: el.innerText,
        tr: el.innerHTML,
      };
    }));
    console.log(tabData);
    tabData.forEach(obj => obj.tag = [head])
    data = data.concat(tabData);
    await page.waitForTimeout(500);
  }
}

scrap();

